package dev.langchain4j.example.crawl;

import dev.langchain4j.example.agent.EmbeddingService;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;

import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

@Service
@Slf4j
public class NewsCrawlerService {
    private final RestTemplate restTemplate;
    private final EmbeddingService embeddingService;

    public NewsCrawlerService(RestTemplate restTemplate, EmbeddingService embeddingService) {
        this.restTemplate = restTemplate;
        this.embeddingService = embeddingService;
    }


    public void crawlAndStoreNews() {
        try {
            // 1. 获取sitemap内容
            String sitemapUrl = "https://news.zgzcw.com/sitemap1.xml?aaa=ss";
            String sitemapContent = restTemplate.getForObject(sitemapUrl, String.class);

            // 2. 解析sitemap获取新闻URL列表
            List<String> newsUrls = parseSitemap(sitemapContent);

            // 3. 遍历URL获取新闻内容
            for (String url : newsUrls) {


                try {
                    String newsContent = fetchNewsContent(url);
                    if (newsContent != null && !newsContent.isEmpty()) {
                        // 5. 使用EmbeddingService处理并存储
                        embeddingService.processDocument(newsContent);
                    }
                } catch (Exception e) {
                    log.error("Error processing news URL: " + url, e);
                }
            }
        } catch (Exception e) {
            log.error("Error crawling news", e);
        }
    }

    private List<String> parseSitemap(String sitemapContent) {
        List<String> urls = new ArrayList<>();
        try {
            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
            DocumentBuilder builder = factory.newDocumentBuilder();
            Document document = builder.parse(new InputSource(new StringReader(sitemapContent)));

            NodeList urlNodes = document.getElementsByTagName("url");
            for (int i = 0; i < urlNodes.getLength(); i++) {
                Node urlNode = urlNodes.item(i);
                NodeList children = urlNode.getChildNodes();
                for (int j = 0; j < children.getLength(); j++) {
                    Node child = children.item(j);
                    if (child.getNodeName().equals("loc")) {
                        urls.add(child.getTextContent());
                    }



                }
            }
        } catch (Exception e) {
            log.error("Error parsing sitemap", e);
        }
        return urls;
    }



    private String fetchNewsContent(String url) {
        try {
            if (url.contains("zx")) {


                // 设置请求头，指定编码
                HttpHeaders headers = new HttpHeaders();
                headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
                headers.set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
                headers.set("Accept-Charset", "UTF-8");

                HttpEntity<String> entity = new HttpEntity<>(headers);
                ResponseEntity<String> response = restTemplate.exchange(url, HttpMethod.GET, entity, String.class);

                // 使用正确的编码解析HTML
                String html = new String(response.getBody().getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);                org.jsoup.nodes.Document doc = Jsoup.parse(html);

                // 获取新闻内容
                Element contentElement = doc.select("body > div.news-main > div.news-main.flex1 > div.news-left2 > div.cont-txt").first();

                if (contentElement != null) {
                    // 获取标题（如果需要）
                    Element titleElement = doc.select("h1").first();
                    String title = titleElement != null ? titleElement.text() : "";

                    // 获取发布时间（如果需要）
                    Element timeElement = doc.select("div.time").first();
                    String publishTime = timeElement != null ? timeElement.text() : "";

                    // 获取正文内容
                    String content = contentElement.text().split("--------------------------------------")[0];

                    // 组合完整信息
                    StringBuilder fullContent = new StringBuilder();
                    if (!title.isEmpty()) {
                        fullContent.append("标题：").append(title).append(",");
                    }

                    fullContent.append("正文：").append(content);

                    // 打印调试信息
                    System.out.println("成功获取到新闻内容，长度: " + fullContent.length());
                    System.out.println("内容预览: " + fullContent.substring(0, Math.min(200, fullContent.length())) + "...");
                    String finalContent = fullContent.toString().replace("声明：足彩网独家稿件，未经授权，禁止转载！", "").trim();
                    if (finalContent.length() < 100) {
                        System.out.println(finalContent);
                    }


                    return finalContent;
                } else {
                    System.out.println("未找到新闻内容，URL: " + url);
                    return null;
                }
            }
            return null;
        } catch (Exception e) {

            log.error("Error fetching news content from: " + url, e);
            return null;
        }
    }


}